import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer;
import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer;

import java.util.Properties;

public class KafkaConnectorSourceDemo {
    public static void main(String[] args) {
        //构建执行任务环境以及任务的启动的入口, 存储全局相关的参数
        //请确保kafka的server.properties中listeners=PLAINTEXT://192.168.123.8:9092有该配置
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        Properties props = new Properties();
        //kafka地址
        props.setProperty("bootstrap.servers", "192.168.123.8:9092");
        //消费者的组名
        props.setProperty("group.id", "video-order-group");
        //字符串序列化和反序列化规则
        props.setProperty("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        props.setProperty("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
        //offset重置规则
        props.setProperty("auto.offset.reset", "latest");
        //自动提交
        props.setProperty("enable.auto.commit", "true");
        props.setProperty("auto.commit.interval.ms", "2000");
        //有后台线程每隔10s检测一下Kafka的分区变化情况
        props.setProperty("flink.partition-discovery.interval-millis","10000");
        FlinkKafkaConsumer<String> consumer =new FlinkKafkaConsumer<>("mytest-topic", new SimpleStringSchema(), props);
        //设置从记录的消费者组内的offset开始消费
        consumer.setStartFromGroupOffsets();
        DataStream<String> ds = env.addSource(consumer);
        ds.print("from kafka(source)");

        //进行transformation，例如每个字符串前+zuikakuedu:
        DataStream<String> outDS = ds.map(new MapFunction<String, String>() {
            @Override
            public String map(String s) throws Exception {
                return "zuikakuedu:"+s;
            }
        });
        outDS.print("to new kafka(sink)");
        //指定kafka的sink，给他推到另一个topic
        outDS.addSink(new FlinkKafkaProducer<String>("myresult-topic",new SimpleStringSchema(),props));


        //DataStream需要调用execute,可以取个名称
        try {
            env.execute("kafka source job");
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
}
